/*
 * @Author: wwssaabb
 * @Date: 2021-07-05 14:32:25
 * @LastEditTime: 2021-07-06 10:36:12
 * @FilePath: \demo\crawler_demo\demo\demo1.js
 */


//爬取豆瓣TOP250电影

const axios=require('axios')
const iconv=require('iconv-lite')
const cheerio=require('cheerio')
const fs=require('fs')
const { last } = require('cheerio/lib/api/traversing')
//{responseType:'arraybuffer'}
//const httpUrl='https://www.dytt8.net/index.htm'
//const httpUrl='https://www.1905.com/vod/list/n_1_t_5/o3.html?fr=vodhome_js_lx'
const httpUrl='https://movie.douban.com/top250'
let films=[]

async function getFilms() {
  let http=films.length===0?httpUrl:films.length===250?'':httpUrl+'?start='+films.length
  if(films.length===250) {
    fs.writeFileSync('films.json',JSON.stringify(films))
    return
  }
  await getFilm(http)
  console.log(films)
  setTimeout(getFilms,3000)
}

async function getFilm(http) {
  let arr=[]
  let title,rating_num,evaluate_num,pic
  return await axios.get(http).then(res=>{
    const $=cheerio.load(res.data)
    $('li .item').each(function(){
      title=$('.title',this).text()
      rating_num=$('.rating_num',this).text()
      evaluate_num=$('.star',this).children().last().text()
      pic=$('.pic img',this).attr('src')
      films.push({title,rating_num,evaluate_num,pic})
    })
    return arr
  })

 
}

getFilms()
